import configmap
import os
import re
import time
import urllib.request
from logutilitie import logutilitie
from publishutilitie import publishutilitie

class webnet:
    """1051*8"""
    domain = ''
    decode = ''
    replaceRule = """<a[^>]*href=["|'](.*?)["|'].*?>(.*?)</a>"""
    #hrefRule = "[a-zA-z]+://[^\s]*[\w$]"
    #hrefRule = "[a-zA-z]+://[^\s]*\.bengou\.com"
    hrefRule = "[a-zA-z]+://www*\.bengou\.com"
    fileterRule = """
    replylist2\.jsp|javascript:
    |\#                                           #need \ for this string
    |http://www\.bengou\.com/zt/61/                 #topic html
    |http://www\.bengou\.com/include/               #include dongman
    |http://www\.bengou\.com/author/                #author info html
    |http://www\.bengou\.com/update/lastweek\.html   #update for lastweek html
    |&page=0
    """
    fileterRule4 = """
    http://www\.bengou\.com/zt/61/                  #topic html
    |http://www\.bengou\.com/update/lastweek\.html   #update for lastweek html
    |http://www\.bengou\.com/include/
    """
    fileterUrlRule4 = """[a-zA-z]+://www\.bengou\.com(.*)/(\d*?)/(\d*?)\.html"""
    hostUrlRule = "(.*)(?<=/)"
    urlFileRule = "\.html|\.htm|\.jsp|\.php|\.asp"
    downPath = os.path.join(os.getcwd(),'down/bengou/html')
    downHtmlNum = 0
    urlListEnd = []
    urlList = set()
    endUrl = ''
    downUrl = ''
    downUrlList = []
    downContent = ''
    readUrlList = []
    log = None
    writeHtmlFile = True
    writeHtmlKey = False
    #----------------------------------------------------------------------
    
    def __init__(self,domain,decode,logInputDirName,writeHtmlFile = False):
        """Constructor"""
        self.domain = domain
        self.decode = decode
        self.downPath = os.path.join(self.downPath,logInputDirName)
        if (os.path.exists(self.downPath) == False):
            os.makedirs(self.downPath)
        self.log = logutilitie(logInputDirName)
        self.writeHtmlFile = writeHtmlFile
        
    #----------------------------------------------------------------------
    
    def run(self,startUrl=''):
        print('start:%s' % publishutilitie.getLocationTime())
        if(len(startUrl) == 0):
            startUrl = self.domain
        start = time.time()
        self.startCacheJob(startUrl)
        #self.log.wirteLog('start_url_list.log',set(self.urlList))
        while len(self.urlList) > 0 :
            url = self.urlList.pop()
            #self.log.wirteLog('all_down.log',url)
            #fileter url for rule
            if(not re.search(self.hrefRule,url) or  re.search(self.fileterRule,url,re.VERBOSE)):
                continue
            #if(re.search(self.fileterUrlRule4,url,re.VERBOSE)):
                #self.log.wirteLog('catch_rule_4.log',url)
                #continue
            end = time.time()
            if( end - start > 15):
                start = end
                print(('time:%s-number:%d-urlnumber:%d' % (publishutilitie.getLocationTime(),self.downHtmlNum,len(self.urlList))))
            #if(self.downHtmlNum == 1):
                #break
            self.endUrl = url
            self.startCacheJob(url)
        print('end:%s' % publishutilitie.getLocationTime())
        return set(self.downUrlList)
    
    #----------------------------------------------------------------------
    
    def startCacheJob(self,url):
        if(not re.search(self.domain,url)):
            #self.log.wirteLog('nodomain.log',url)
            return
        if(url in self.urlListEnd):
            #self.log.wirteLog('same_url.log',url)
            return
        if(1):
        #try:
            self.urlListEnd = [url]
            connect = self.getHtmlFromUrl(url)
            if(len( connect )  == 0):
                #self.log.wirteLog('null_error.log',url)
                return
            #write html to file
            self.getUrlFromHtml(connect)
        #except ValueError:
            #self.log.wirteLog('value_error.log',url)
            #return
        #except :
            #self.log.wirteLog('error.log',url)
            #return
    
    #----------------------------------------------------------------------
    
    def getHtmlFromUrl(self,url):
        """"""
        self.downContent = ''
        if(url in self.readUrlList):
            return self.downContent
        self.writeHtmlKey = False
        #print('down -' + url)
        self.downUrl = url
        self.readUrlList = self.readUrlList + [url]
        try:
            response = urllib.request.urlopen(url)
            self.downContent = response.read()
            self.log.wirteLog('down.log',url)
        except :
            self.log.wirteLog('error.log',url)
            return self.downContent
        if(self.decode):
            self.downContent = self.downContent.decode(self.decode)
        return self.downContent;
        
    
    #----------------------------------------------------------------------
    
    def getHtmlToFile(self,fileName,content,writeType='w'):
        """"""
        filePath = os.path.join(self.downPath,fileName)
        f=open(filePath,writeType, encoding=self.decode)
        f.write(content)
        f.close()
        return filePath
    
    #----------------------------------------------------------------------
    
    def getUrlFromHtml(self,content):
        """getUrlFromHtml"""
        lineInfoList = re.findall(self.replaceRule,content)
        urlHrefList = []
        for lineInfo in lineInfoList:
            url = self.getAbsoluteUrl(lineInfo[0],lineInfo[1])
            if(len(url) > 0 ):
                urlHrefList = urlHrefList + [url]
        self.urlList = self.urlList.union(urlHrefList)

    
    #----------------------------------------------------------------------
    
    def getAbsoluteUrl(self,url,urlInfo):
        
        if(re.match('^/(.*?)',url)):
            url = '%s%s' % tuple([self.domain,url])
            #self.log.wirteLog('rule_1.log',url)
        elif(re.match('^http://',url)):
            url = url
            #self.log.wirteLog('rule_2.log',url)
        elif(re.match(self.fileterRule,url,re.VERBOSE)):
            url = url
            #self.log.wirteLog('rule_3.log',url)
        elif(re.search(self.urlFileRule,url)):
            m = re.match(self.hostUrlRule,self.downUrl)
            url = '%s%s' % tuple([m.group(0),url])
            if(not re.match(self.fileterRule4,url,re.VERBOSE)):
                self.log.wirteLog('rule_4.log',url)
                url = '' 
                if(not self.downUrl in self.downUrlList):
                    self.downHtmlNum += 1
                    self.getHtmlToFile("%d.html" % self.downHtmlNum,self.downContent)
                    logContent = "%d%s%s" % tuple([self.downHtmlNum,configmap.splitLogString,self.downUrl])
                    self.log.wirteLog('down_url_for_have_rule_4.log',logContent)
                    self.downUrlList = self.downUrlList + [self.downUrl]
        return url
    
    #----------------------------------------------------------------------
    
    
    
    def __del__(self):
        print('webnet__del__info')
        print(self.endUrl)
        print(self.downHtmlNum)
        if(len(self.urlListEnd) > 0):
            self.log.wirteLog('urlListEnd.log',self.urlListEnd)
        if(len(self.urlList) > 0):
            self.log.wirteLog('urlList.log',self.urlList)
        print('webnet__del__info')
        
        
        
if(__name__=='__main__'):
    logInputDirName = input('Enter log dir name --> ')
    webnet(configmap.WebTargetUrl,configmap.WebDecode,logInputDirName).run('http://www.bengou.com/folder/folderList2.jsp?letter=A&demo=1&page=1')

